# -*- coding: utf-8 -*-
# @Time    : 2024/5/30 13:36
# @Author  : Alvin
# @File    : extensions.py
# extensions.py
from scrapy import signals
from scrapy.utils.log import failure_to_exc_info
from twisted.internet.error import TimeoutError, DNSLookupError
from twisted.python import log as twisted_log
from datetime import datetime
import pymysql
import os
GLOBAL_HOST = os.getenv('GLOBAL_HOST', 'localhost')

class GlobalErrorHandlingExtension:
    def __init__(self, stats):
        self.stats = stats
        self.db = None  # 数据库连接初始化将在这里完成
        self.job_id = None

    @classmethod
    def from_crawler(cls, crawler):
        ext = cls(crawler.stats)
        crawler.signals.connect(ext.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(ext.spider_closed, signal=signals.spider_closed)
        # 连接我们的自定义日志观察者
        twisted_log.addObserver(ext.log_error)
        return ext

    def spider_opened(self, spider):
        # 存储当前爬虫的 job_id
        self.job_id= spider.job_id
        spider.logger.info(f"this is job_id:{self.job_id}")

    def spider_closed(self, spider, reason):
        # 爬虫关闭时执行的操作
        pass  # 根据需要实现

    def log_error(self, event, error):
        # 处理 Twisted 记录的日志错误
        if (event == 'unhandledError' or event == 'log.err') and isinstance(error.value, ModuleNotFoundError):
            self._update_db_status(self.job_id,  error)  # 根据需要更新数据库状态

    def _update_db_status(self, job_id, error):
        # 更新数据库状态
        if not self.db:
            # 初始化数据库连接
            self.db = pymysql.connect(host=GLOBAL_HOST,
                                      user='remoteuser',
                                      password='123456',
                                      db='auto_scrapy',
                                      cursorclass=pymysql.cursors.DictCursor)
        # 执行数据库更新操作
        with self.db.cursor() as cursor:
            sql = "UPDATE task_records SET end_time=%s, status=%s WHERE task_id=%s"
            cursor.execute(sql, (datetime.utcnow(), 'failed', job_id))
            self.db.commit()
            # 记录错误详情到数据库
            error_msg = str(error)
            cursor.execute("INSERT INTO errors (task_id, error_msg, timestamp) VALUES (%s, %s, %s)",
                           (job_id, error_msg, datetime.utcnow()))
            self.db.commit()

    def close(self):
        # 关闭数据库连接
        if self.db:
            self.db.close()
            # 移除日志观察者
            twisted_log.removeObserver(self.log_error)

